"""微博数据清洗管道"""
import re
import html
import logging
from typing import Dict, Any, List, Optional
from datetime import datetime
import unicodedata
from ..security.input_sanitizer import input_sanitizer

logger = logging.getLogger(__name__)


class WeiboCleaningPipeline:
    """微博数据清洗管道
    
    负责清理HTML标签、特殊字符、格式化时间等
    """
    
    def __init__(self):
        """初始化清洗管道"""
        self.sensitive_words = [
            # 可配置敏感词列表
        ]
        
    def process(self, item: Dict[str, Any]) -> Optional[Dict[str, Any]]:
        """处理单个数据项
        
        Args:
            item: 原始数据项
            
        Returns:
            清洗后的数据项，无效数据返回None
        """
        try:
            # 安全验证 - 检测注入攻击
            if not input_sanitizer.validate_data_item(item):
                logger.warning("数据项包含可疑内容，已拒绝")
                return None
            
            # 验证必要字段
            if not self._validate_item(item):
                return None
            
            # 使用安全清理器处理数据
            item = input_sanitizer.sanitize_data_item(item)
                
            # 清理标题
            if 'title' in item:
                item['title'] = self.clean_text(item['title'])
                
            # 清理详情内容
            if 'content' in item and isinstance(item['content'], dict):
                if 'description' in item['content']:
                    item['content']['description'] = self.clean_text(
                        item['content']['description']
                    )
                    
                # 清理评论
                if 'comments' in item['content']:
                    item['content']['comments'] = self._clean_comments(
                        item['content']['comments']
                    )
                    
            # 清理URL字段
            if 'url' in item:
                item['url'] = input_sanitizer.sanitize_url(item['url'])
            if 'detail_url' in item:
                item['detail_url'] = input_sanitizer.sanitize_url(item['detail_url'])
                
            # 格式化时间
            if 'fetch_time' in item:
                item['fetch_time'] = self.format_time(item['fetch_time'])
                
            # 添加更新时间
            item['update_time'] = datetime.utcnow().isoformat() + 'Z'
            
            # 过滤敏感信息
            if self._contains_sensitive(item):
                logger.warning(f"数据包含敏感信息，已过滤: {item.get('title', '')}")
                return None
                
            return item
            
        except Exception as e:
            logger.error(f"清洗数据失败: {e}")
            return None
            
    def clean_text(self, text: str) -> str:
        """清理文本内容
        
        Args:
            text: 原始文本
            
        Returns:
            清理后的文本
        """
        if not text:
            return ""
            
        # 解码HTML实体
        text = html.unescape(text)
        
        # 移除HTML标签
        text = re.sub(r'<[^>]+>', '', text)
        
        # 移除多余的空白字符
        text = re.sub(r'\s+', ' ', text)
        
        # 处理特殊Unicode字符
        text = self._normalize_unicode(text)
        
        # 处理微博表情
        text = self._process_emoji(text)
        
        # 移除首尾空白
        text = text.strip()
        
        return text
        
    def _normalize_unicode(self, text: str) -> str:
        """标准化Unicode字符
        
        Args:
            text: 原始文本
            
        Returns:
            标准化后的文本
        """
        # 将组合字符规范化
        text = unicodedata.normalize('NFC', text)
        
        # 移除零宽字符
        zero_width_chars = [
            '\u200b',  # 零宽空格
            '\u200c',  # 零宽非连接符
            '\u200d',  # 零宽连接符
            '\ufeff',  # 零宽非断空格
        ]
        for char in zero_width_chars:
            text = text.replace(char, '')
            
        return text
        
    def _process_emoji(self, text: str) -> str:
        """处理微博表情
        
        Args:
            text: 包含表情的文本
            
        Returns:
            处理后的文本
        """
        # 保留常见emoji，移除微博特殊表情标记
        text = re.sub(r'\[([^\]]+)\]', r'[\1]', text)  # 保留方括号表情
        
        # 移除图片表情链接
        text = re.sub(r'http://[^\s]*\.(?:jpg|png|gif)', '', text)
        
        return text
        
    def format_time(self, time_str: str) -> str:
        """格式化时间为ISO格式
        
        Args:
            time_str: 原始时间字符串
            
        Returns:
            ISO格式时间字符串
        """
        if not time_str:
            return datetime.utcnow().isoformat() + 'Z'
            
        # 如果已经是ISO格式，直接返回
        if 'T' in time_str and time_str.endswith('Z'):
            return time_str
            
        # 处理微博时间格式
        time_formats = [
            '%Y-%m-%d %H:%M:%S',
            '%Y-%m-%d %H:%M',
            '%Y年%m月%d日 %H:%M',
            '%m月%d日 %H:%M',
        ]
        
        for fmt in time_formats:
            try:
                dt = datetime.strptime(time_str, fmt)
                # 如果只有月日，补充年份
                if dt.year == 1900:
                    dt = dt.replace(year=datetime.now().year)
                return dt.isoformat() + 'Z'
            except (ValueError, TypeError) as e:
                logger.debug(f"时间格式 {fmt} 解析失败: {time_str}, 错误: {e}")
                continue
                
        # 处理"刚刚"、"x分钟前"等相对时间
        if '刚刚' in time_str:
            return datetime.utcnow().isoformat() + 'Z'
        elif '分钟前' in time_str:
            minutes = int(re.search(r'(\d+)分钟前', time_str).group(1))
            dt = datetime.utcnow() - timedelta(minutes=minutes)
            return dt.isoformat() + 'Z'
        elif '小时前' in time_str:
            hours = int(re.search(r'(\d+)小时前', time_str).group(1))
            dt = datetime.utcnow() - timedelta(hours=hours)
            return dt.isoformat() + 'Z'
            
        # 无法解析，返回当前时间
        logger.warning(f"无法解析时间格式: {time_str}")
        return datetime.utcnow().isoformat() + 'Z'
        
    def _clean_comments(self, comments: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        """清理评论列表
        
        Args:
            comments: 原始评论列表
            
        Returns:
            清理后的评论列表
        """
        cleaned_comments = []
        
        for comment in comments:
            if not comment.get('content'):
                continue
                
            cleaned_comment = {
                'author': self.clean_text(comment.get('author', '')),
                'content': self.clean_text(comment.get('content', '')),
                'likes': comment.get('likes', 0),
                'time': self.format_time(comment.get('time', ''))
            }
            
            # 过滤空评论
            if cleaned_comment['content']:
                cleaned_comments.append(cleaned_comment)
                
        return cleaned_comments
        
    def _validate_item(self, item: Dict[str, Any]) -> bool:
        """验证数据项是否有效
        
        Args:
            item: 数据项
            
        Returns:
            是否有效
        """
        # 必需字段
        required_fields = ['title', 'url', 'source']
        
        for field in required_fields:
            if field not in item or not item[field]:
                logger.warning(f"缺少必需字段: {field}")
                return False
                
        # 验证URL格式
        if not item['url'].startswith(('http://', 'https://')):
            logger.warning(f"无效的URL: {item['url']}")
            return False
            
        # 验证source
        if item['source'] != 'weibo':
            logger.warning(f"无效的source: {item['source']}")
            return False
            
        return True
        
    def _contains_sensitive(self, item: Dict[str, Any]) -> bool:
        """检查是否包含敏感信息
        
        Args:
            item: 数据项
            
        Returns:
            是否包含敏感信息
        """
        # 检查标题
        title = item.get('title', '').lower()
        for word in self.sensitive_words:
            if word.lower() in title:
                return True
                
        # 检查描述
        if 'content' in item and 'description' in item['content']:
            desc = item['content']['description'].lower()
            for word in self.sensitive_words:
                if word.lower() in desc:
                    return True
                    
        return False
        
    def process_batch(self, items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        """批量处理数据项
        
        Args:
            items: 数据项列表
            
        Returns:
            清洗后的数据项列表
        """
        cleaned_items = []
        
        for item in items:
            cleaned_item = self.process(item)
            if cleaned_item:
                cleaned_items.append(cleaned_item)
                
        logger.info(f"清洗完成: 输入{len(items)}条，输出{len(cleaned_items)}条")
        return cleaned_items


from datetime import timedelta  # 添加缺失的导入